{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CLIENT_ID = \"hZ-F6QLaGJ6VBg\" \n",
    "CLIENT_SECRET = \"MFC8FXTlXjm4jtG60bm9Qs-PPls\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "USER_AGENT = \"python:<test0.1> (by /u/Dealsy23)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "USERNAME = \"Dealsy23\" \n",
    "PASSWORD = \"Outbreak23\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "def login(username, password):\n",
    "    if password is None:\n",
    "        password = getpass.getpass(\"Enter reddit password for user {}: \".format(username))    \n",
    "    headers = {\"User-Agent\": USER_AGENT}\n",
    "    # Setup an auth object with our credentials\n",
    "    client_auth = requests.auth.HTTPBasicAuth(CLIENT_ID, CLIENT_SECRET)\n",
    "    # Make a post request to the access_token endpoint\n",
    "    post_data = {\"grant_type\": \"password\", \"username\": username, \"password\": password}\n",
    "    response = requests.post(\"https://www.reddit.com/api/v1/access_token\", auth=client_auth,     \n",
    "                             data=post_data, headers=headers) \n",
    "    return response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "token = login(USERNAME, PASSWORD)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'access_token': 'K9NuIv1-0S3zDIVj6oyOD1dw83Y',\n",
       " 'expires_in': 3600,\n",
       " 'scope': '*',\n",
       " 'token_type': 'bearer'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subreddit = \"worldnews\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = \"https://oauth.reddit.com/r/{}\".format(subreddit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "headers = {\"Authorization\": \"bearer {}\".format(token['access_token']), \n",
    "\"User-Agent\": USER_AGENT}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "response = requests.get(url, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "result = response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Russia considers banning sale of cigarettes to anyone born after 2015\n",
      "Swiss Muslim girls must swim with boys\n",
      "Report: Russia spread fake news and disinformation in Sweden - Russia has coordinated a campaign over the past 2years to influence Sweden’s decision making by using disinformation, propaganda and false documents, according to a report by researchers at The Swedish Institute of International Affairs.\n",
      "100% of Dutch Trains Now Run on Wind Energy. The Netherlands met its renewable energy goals a year ahead of time.\n",
      "Legal challenge against UK’s sweeping surveillance laws quickly crowdfunded\n",
      "A 1,000-foot-thick ice block about the size of Delaware is snapping off of Antarctica\n",
      "The U.S. dropped an average of 72 bombs every day — the equivalent of three an hour — in 2016, according to an analysis of American strikes around the world. U.S. Bombed Iraq, Syria, Pakistan, Afghanistan, Libya, Yemen, Somalia in 2016\n",
      "The German government is investigating a recent surge in fake news following claims that Russia is attempting to meddle in the country’s parliamentary elections later this year.\n",
      "Pesticides kill over 10 million bees in a matter of days in Brazil countryside\n",
      "The families of American victims of Islamic State terrorist attacks in Europe have sued Twitter, charging that the social media giant allowed the terror group to proliferate online\n",
      "Gas taxes drop globally despite climate change; oil &amp; gas industry gets $500 billion in subsidies; last new US gas tax was in 1993\n",
      "Czech government tells citizens to arm themselves and shoot Muslim terrorists in case of 'Super Holocaust'\n",
      "PLO threatens to revoke recognition of Israel if US embassy moves to Jerusalem\n",
      "Two-thirds of all new HIV cases in Europe are being recorded in just one country – Russia: More than a million Russians now live with the virus and that number is expected to nearly double in the next decade\n",
      "Czech government tells its citizens how to fight terrorists: Shoot them yourselves | The interior ministry is pushing a constitutional change that would let citizens use guns against terrorists\n",
      "Morocco Prohibits Sale of Burqa\n",
      "Mass killer Breivik makes Nazi salute at rights appeal case\n",
      "Soros Groups Risk Purge After Trump’s Win Emboldens Hungary\n",
      "Nigeria purges 50,000 ‘ghost workers’ from State payroll in corruption sweep\n",
      "Alcohol advertising is aggressive and linked to youth drinking, research finds | Society\n",
      "UK Government quietly launched ‘assault on freedom’ while distracting people, say campaigners behind legal challenge - The Investigatory Powers Act became law at the end of last year, and gives spies the power to read through everyone’s entire internet history\n",
      "Russia’s Reserve Fund down 70 percent in 2016\n",
      "Russian diplomat found dead in Athens\n",
      "At least 21 people have been killed (most were civilians) and 45 wounded in twin bombings near the Afghan parliament in Kabul\n",
      "Pound’s Decline Deepens as Currency Reclaims Dubious Honor\n"
     ]
    }
   ],
   "source": [
    "for story in result['data']['children']: \n",
    "    print(story['data']['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_links(subreddit, token, n_pages=5):\n",
    "    stories = []\n",
    "    after = None\n",
    "    for page_number in range(n_pages):\n",
    "        # Sleep before making calls to avoid going over the API limit\n",
    "        sleep(2)\n",
    "        # Setup headers and make call, just like in the login function\n",
    "        headers = {\"Authorization\": \"bearer {}\".format(token['access_token']), \"User-Agent\": USER_AGENT} \n",
    "        url = \"https://oauth.reddit.com/r/{}?limit=100\". format(subreddit)\n",
    "        if after:\n",
    "            # Append cursor for next page, if we have one\n",
    "            url += \"&after={}\".format(after)\n",
    "        response = requests.get(url, headers=headers)\n",
    "        result = response.json()\n",
    "        # Get the new cursor for the next loop\n",
    "        after = result['data']['after']\n",
    "        # Add all of the news items to our stories list\n",
    "        for story in result['data']['children']:\n",
    "            stories.append((story['data']['title'], story['data']['url'], story['data']['score']))\n",
    "    return stories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stories = get_links(\"worldnews\", token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os \n",
    "data_folder = os.path.join(\"data\", \"websites\", \"raw\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "number_errors = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for title, url, score in stories:\n",
    "    output_filename = hashlib.md5(url.encode()).hexdigest() \n",
    "    fullpath = os.path.join(data_folder, output_filename + \".txt\")\n",
    "    try: \n",
    "        response = requests.get(url) \n",
    "        data = response.text \n",
    "        with open(fullpath, 'w') as outf: \n",
    "            outf.write(data)\n",
    "    except Exception as e:\n",
    "        number_errors += 1\n",
    "        # You can use this to view the errors, if you are getting too many:\n",
    "        #raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "503"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "number_errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "filenames = [os.path.join(data_folder, filename) for filename in os.listdir(data_folder)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_output_folder = os.path.join(\"data\", \"websites\", \"textonly\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import lxml\n",
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "skip_node_types = [\"script\", \"head\", \"style\", etree.Comment]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parser = etree.HTMLParser()\n",
    "\n",
    "def get_text_from_file(filename):\n",
    "    with open(filename) as inf:\n",
    "        html_tree = etree.parse(inf, parser) \n",
    "    return get_text_from_node(html_tree.getroot())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_text_from_node(node):\n",
    "    if len(node) == 0: \n",
    "        # No children, just return text from this item\n",
    "        if node.text: \n",
    "            return node.text \n",
    "        else:\n",
    "            return \"\"\n",
    "    else:\n",
    "        # This node has children, return the text from it:\n",
    "        results = (get_text_from_node(child)\n",
    "                   for child in node\n",
    "                   if child.tag not in skip_node_types)\n",
    "    result = str.join(\"\\n\", (r for r in results if len(r) > 1))\n",
    "    if len(result) >= 100:\n",
    "        return result\n",
    "    else:\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for filename in os.listdir(data_folder):\n",
    "    text = get_text_from_file(os.path.join(data_folder, filename)) \n",
    "    with open(os.path.join(text_output_folder, filename), 'w') as outf: \n",
    "        outf.write(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "n_clusters = 10 \n",
    "pipeline = Pipeline([('feature_extraction', TfidfVectorizer(max_df=0.4)),\n",
    "                                     ('clusterer', KMeans(n_clusters=n_clusters)) ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "documents = [open(os.path.join(text_output_folder, filename)).read() for filename in os.listdir(text_output_folder)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pipeline.fit(documents)\n",
    "labels = pipeline.predict(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0 contains 42 samples\n",
      "Cluster 1 contains 37 samples\n",
      "Cluster 2 contains 31 samples\n",
      "Cluster 3 contains 26 samples\n",
      "Cluster 4 contains 28 samples\n",
      "Cluster 5 contains 13 samples\n",
      "Cluster 6 contains 43 samples\n",
      "Cluster 7 contains 25 samples\n",
      "Cluster 8 contains 209 samples\n",
      "Cluster 9 contains 37 samples\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "c = Counter(labels) \n",
    "for cluster_number in range(n_clusters): \n",
    "    print(\"Cluster {} contains {} samples\".format(cluster_number, c[cluster_number]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "414.5540395641533"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.named_steps['clusterer'].inertia_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "inertia_scores = [] \n",
    "n_cluster_values = list(range(2, 20)) \n",
    "for n_clusters in n_cluster_values: \n",
    "    cur_inertia_scores = [] \n",
    "    X = TfidfVectorizer(max_df=0.4).fit_transform(documents) \n",
    "    for i in range(10): \n",
    "        km = KMeans(n_clusters=n_clusters).fit(X) \n",
    "        cur_inertia_scores.append(km.inertia_) \n",
    "        inertia_scores.append(cur_inertia_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_clusters = 6 \n",
    "pipeline = Pipeline([('feature_extraction', TfidfVectorizer(max_df=0.4)),\n",
    "                     ('clusterer', KMeans(n_clusters=n_clusters)) ])\n",
    "pipeline.fit(documents) \n",
    "labels = pipeline.predict(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "terms = pipeline.named_steps['feature_extraction'].get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = Counter(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0 contains 22 samples\n",
      " Most important terms\n",
      " 1) iran (score: 0.1932)\n",
      " 2) iranian (score: 0.1310)\n",
      " 3) shots (score: 0.1198)\n",
      " 4) mahan (score: 0.1109)\n",
      " 5) navy (score: 0.1108)\n",
      "Cluster 1 contains 36 samples\n",
      " Most important terms\n",
      " 1) click (score: 0.1833)\n",
      " 2) adblock (score: 0.1786)\n",
      " 3) icon (score: 0.1166)\n",
      " 4) site (score: 0.1012)\n",
      " 5) getty (score: 0.0796)\n",
      "Cluster 2 contains 308 samples\n",
      " Most important terms\n",
      " 1) she (score: 0.0185)\n",
      " 2) her (score: 0.0178)\n",
      " 3) china (score: 0.0174)\n",
      " 4) us (score: 0.0171)\n",
      " 5) government (score: 0.0141)\n",
      "Cluster 3 contains 43 samples\n",
      " Most important terms\n",
      " 1) kabul (score: 0.0654)\n",
      " 2) afghanistan (score: 0.0566)\n",
      " 3) taliban (score: 0.0558)\n",
      " 4) killed (score: 0.0509)\n",
      " 5) pakistan (score: 0.0490)\n",
      "Cluster 4 contains 45 samples\n",
      " Most important terms\n",
      " 1) washington (score: 0.1076)\n",
      " 2) industry (score: 0.0755)\n",
      " 3) tuesday (score: 0.0742)\n",
      " 4) your (score: 0.0719)\n",
      " 5) tax (score: 0.0699)\n",
      "Cluster 5 contains 37 samples\n",
      " Most important terms\n",
      " 1) israeli (score: 0.1896)\n",
      " 2) israel (score: 0.1494)\n",
      " 3) jerusalem (score: 0.1114)\n",
      " 4) soldiers (score: 0.0775)\n",
      " 5) embassy (score: 0.0769)\n"
     ]
    }
   ],
   "source": [
    "for cluster_number in range(n_clusters): \n",
    "    print(\"Cluster {} contains {} samples\".format(cluster_number, c[cluster_number]))\n",
    "    print(\" Most important terms\")\n",
    "    centroid = pipeline.named_steps['clusterer'].cluster_centers_[cluster_number]\n",
    "    most_important = centroid.argsort()\n",
    "    for i in range(5):\n",
    "        term_index = most_important[-(i+1)]\n",
    "        print(\" {0}) {1} (score: {2:.4f})\".format(i+1, terms[term_index], centroid[term_index]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = pipeline.transform(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def create_coassociation_matrix(labels):\n",
    "    rows = [] \n",
    "    cols = []\n",
    "    unique_labels = set(labels) \n",
    "    for label in unique_labels:\n",
    "        indices = np.where(labels == label)[0]\n",
    "        for index1 in indices:\n",
    "            for index2 in indices:\n",
    "                rows.append(index1)\n",
    "                cols.append(index2)\n",
    "    data = np.ones((len(rows),)) \n",
    "    return csr_matrix((data, (rows, cols)), dtype='float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "C = create_coassociation_matrix(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<491x491 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 101887 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse.csgraph import minimum_spanning_tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mst = minimum_spanning_tree(C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mst = minimum_spanning_tree(-C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline.fit(documents) \n",
    "labels2 = pipeline.predict(documents) \n",
    "C2 = create_coassociation_matrix(labels2) \n",
    "C_sum = (C + C2) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mst = minimum_spanning_tree(-C_sum) \n",
    "mst.data[mst.data > -1] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse.csgraph import connected_components \n",
    "number_of_clusters, labels = connected_components(mst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, ClusterMixin \n",
    "class EAC(BaseEstimator, ClusterMixin):\n",
    "    def __init__(self, n_clusterings=10, cut_threshold=0.5, n_clusters_range=(3, 10)): \n",
    "        self.n_clusterings = n_clusterings\n",
    "        self.cut_threshold = cut_threshold\n",
    "        self.n_clusters_range = n_clusters_range\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        C = sum((create_coassociation_matrix(self._single_clustering(X))\n",
    "                 for i in range(self.n_clusterings)))\n",
    "        mst = minimum_spanning_tree(-C)\n",
    "        mst.data[mst.data > -self.cut_threshold] = 0\n",
    "        mst.eliminate_zeros()\n",
    "        self.n_components, self.labels_ = connected_components(mst)\n",
    "        return self\n",
    " \n",
    "    def _single_clustering(self, X):\n",
    "        n_clusters = np.random.randint(*self.n_clusters_range)\n",
    "        km = KMeans(n_clusters=n_clusters)\n",
    "        return km.fit_predict(X)\n",
    " \n",
    "    def fit_predict(self, X):\n",
    "        self.fit(X)\n",
    "        return self.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([('feature_extraction', TfidfVectorizer(max_df=0.4)),\n",
    "                     ('clusterer', EAC()) ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec = TfidfVectorizer(max_df=0.4) \n",
    "X = vec.fit_transform(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import MiniBatchKMeans \n",
    "mbkm = MiniBatchKMeans(random_state=14, n_clusters=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 10 \n",
    "for iteration in range(int(X.shape[0] / batch_size)): \n",
    "    start = batch_size * iteration \n",
    "    end = batch_size * (iteration + 1) \n",
    "    mbkm.partial_fit(X[start:end])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = mbkm.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2,\n",
       "       2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2], dtype=int32)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PartialFitPipeline(Pipeline):\n",
    "    def partial_fit(self, X, y=None):\n",
    "        Xt = X\n",
    "        for name, transform in self.steps[:-1]:\n",
    "            Xt = transform.transform(Xt)\n",
    "        return self.steps[-1][1].partial_fit(Xt, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "\n",
    "pipeline = PartialFitPipeline([('feature_extraction', HashingVectorizer()),\n",
    "                               ('clusterer', MiniBatchKMeans(random_state=14, n_clusters=3)) ])\n",
    "batch_size = 10 \n",
    "for iteration in range(int(len(documents) / batch_size)): \n",
    "    start = batch_size * iteration \n",
    "    end = batch_size * (iteration + 1)\n",
    "    pipeline.partial_fit(documents[start:end]) \n",
    "labels = pipeline.predict(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, 0, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2,\n",
       "       1, 0, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1,\n",
       "       1, 2, 0, 1, 0, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 0, 1, 1, 1, 0, 1,\n",
       "       1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 2, 1, 1, 2, 0, 1, 1, 1, 1, 2, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 0, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 0, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 2, 1, 1,\n",
       "       2, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 0, 1, 1, 0,\n",
       "       2, 1, 1, 1, 0, 0, 1, 2, 1, 1, 1, 0, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 0, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 0, 1, 1, 2, 2, 1, 1, 2,\n",
       "       2, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       2, 2, 1, 2, 1, 0, 1, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, 1, 1, 1, 2, 1, 1,\n",
       "       1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 2, 1, 1, 0, 1, 1, 1, 1,\n",
       "       0, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 2, 1, 1, 2, 1, 1, 1, 1, 1, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1], dtype=int32)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
